{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ba5c49e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import os\n",
    "from typing import List, Dict, Any\n",
    "from dotenv import load_dotenv\n",
    "from code_ingestion import ingest_github_repo\n",
    "from model_service import get_model_response_async\n",
    "from code_evaluation import evaluate_code\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "async def get_full_response(model_name: str, prompt: str, context: Dict[str, Any]) -> str:\n",
    "    \"\"\"\n",
    "    Get a complete response from a model (non-streaming)\n",
    "    \"\"\"\n",
    "    response_text = \"\"\n",
    "    async for chunk in get_model_response_async(model_name, prompt, context):\n",
    "        response_text += chunk\n",
    "    \n",
    "    # Clean up the response by removing markdown code blocks\n",
    "    cleaned_text = response_text.strip().removeprefix(\"```python\").removeprefix(\"```\").removesuffix(\"```\").strip()\n",
    "    return cleaned_text\n",
    "\n",
    "async def process_single_query(query: str, context: Dict[str, Any]) -> Dict[str, Any]:\n",
    "    \"\"\"\n",
    "    Process a single query and return results for both models\n",
    "    \"\"\"\n",
    "    try:\n",
    "        # Get responses from both models\n",
    "        claude_response, qwen_response = await asyncio.gather(\n",
    "            get_full_response(\"claude-4\", query, context),\n",
    "            get_full_response(\"qwen3-coder\", query, context)\n",
    "        )\n",
    "        \n",
    "        # Evaluate both responses\n",
    "        claude_eval = evaluate_code(claude_response)\n",
    "        qwen_eval = evaluate_code(qwen_response)\n",
    "        \n",
    "        # Return results in the specified format\n",
    "        return {\n",
    "            \"query\": query,\n",
    "            \"Correctness_Qwen\": qwen_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n",
    "            \"Correctness_Sonnet\": claude_eval[\"detailed_metrics\"][\"correctness\"][\"score\"],\n",
    "            \"readability_Qwen\": qwen_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n",
    "            \"readability_Sonnet\": claude_eval[\"detailed_metrics\"][\"readability\"][\"score\"],\n",
    "            \"bestpractices_Qwen\": qwen_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n",
    "            \"bestpractices_Sonnet\": claude_eval[\"detailed_metrics\"][\"best_practices\"][\"score\"],\n",
    "            # Additional fields for debugging/analysis\n",
    "            \"claude_response\": claude_response,\n",
    "            \"qwen_response\": qwen_response,\n",
    "            \"claude_overall_score\": claude_eval[\"overall_score\"],\n",
    "            \"qwen_overall_score\": qwen_eval[\"overall_score\"]\n",
    "        }\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"Error processing query '{query}': {str(e)}\")\n",
    "        return {\n",
    "            \"query\": query,\n",
    "            \"Correctness_Qwen\": 0.0,\n",
    "            \"Correctness_Sonnet\": 0.0,\n",
    "            \"readability_Qwen\": 0.0,\n",
    "            \"readability_Sonnet\": 0.0,\n",
    "            \"bestpractices_Qwen\": 0.0,\n",
    "            \"bestpractices_Sonnet\": 0.0,\n",
    "            \"error\": str(e)\n",
    "        }\n",
    "\n",
    "async def process_queries(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n",
    "    \"\"\"\n",
    "    Process a list of queries against a GitHub repository\n",
    "    \n",
    "    Args:\n",
    "        queries: List of query strings\n",
    "        github_repo_url: URL of the GitHub repository to ingest\n",
    "        \n",
    "    Returns:\n",
    "        List of dictionaries with evaluation results for each query\n",
    "    \"\"\"\n",
    "    # Ingest the repository to get context\n",
    "    print(f\"Ingesting repository: {github_repo_url}\")\n",
    "    context = ingest_github_repo(github_repo_url)\n",
    "    print(\"Repository ingested successfully!\")\n",
    "    \n",
    "    # Process all queries\n",
    "    results = []\n",
    "    for i, query in enumerate(queries, 1):\n",
    "        print(f\"Processing query {i}/{len(queries)}: {query[:50]}...\")\n",
    "        result = await process_single_query(query, context)\n",
    "        results.append(result)\n",
    "        print(result)\n",
    "        print(f\"Completed query {i}/{len(queries)}\")\n",
    "    \n",
    "    return results\n",
    "\n",
    "def process_queries_sync(queries: List[str], github_repo_url: str) -> List[Dict[str, Any]]:\n",
    "    \"\"\"\n",
    "    Synchronous wrapper for process_queries - works in Jupyter notebooks\n",
    "    \"\"\"\n",
    "    try:\n",
    "        # Try to get the current event loop\n",
    "        loop = asyncio.get_event_loop()\n",
    "        if loop.is_running():\n",
    "            # If we're in a running event loop (like Jupyter), use asyncio.create_task\n",
    "            import nest_asyncio\n",
    "            nest_asyncio.apply()\n",
    "            return loop.run_until_complete(process_queries(queries, github_repo_url))\n",
    "        else:\n",
    "            # If no event loop is running, use asyncio.run\n",
    "            return asyncio.run(process_queries(queries, github_repo_url))\n",
    "    except RuntimeError:\n",
    "        # Fallback for Jupyter notebooks\n",
    "        import nest_asyncio\n",
    "        nest_asyncio.apply()\n",
    "        loop = asyncio.get_event_loop()\n",
    "        return loop.run_until_complete(process_queries(queries, github_repo_url))\n",
    "\n",
    "# Example usage:\n",
    "if True:\n",
    "    # Define your queries\n",
    "    mcp_prompts = [\n",
    "    \"Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.\",\n",
    "    \"Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.\",\n",
    "    \"Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.\",\n",
    "    \"Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.\",\n",
    "    \"Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.\",\n",
    "    \"Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.\",\n",
    "    \"Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.\",\n",
    "    \"Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.\",\n",
    "    \"Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.\",\n",
    "    \"Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.\"\n",
    "]\n",
    "    \n",
    "    # Define the GitHub repository URL\n",
    "    repo_url = \"https://github.com/jlowin/fastmcp\"\n",
    "    \n",
    "    # Process all queries\n",
    "    results = process_queries_sync(mcp_prompts, repo_url)\n",
    "    \n",
    "    # Print results\n",
    "    for result in results:\n",
    "        print(f\"\\nQuery: {result['query']}\")\n",
    "        print(f\"Qwen Correctness: {result['Correctness_Qwen']:.2f}\")\n",
    "        print(f\"Sonnet Correctness: {result['Correctness_Sonnet']:.2f}\")\n",
    "        print(f\"Qwen Readability: {result['readability_Qwen']:.2f}\")\n",
    "        print(f\"Sonnet Readability: {result['readability_Sonnet']:.2f}\")\n",
    "        print(f\"Qwen Best Practices: {result['bestpractices_Qwen']:.2f}\")\n",
    "        print(f\"Sonnet Best Practices: {result['bestpractices_Sonnet']:.2f}\")\n",
    "        if 'error' in result:\n",
    "            print(f\"Error: {result['error']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f14722e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_339c8_row0_col0, #T_339c8_row0_col1, #T_339c8_row0_col2, #T_339c8_row0_col3, #T_339c8_row0_col4, #T_339c8_row0_col5, #T_339c8_row0_col6, #T_339c8_row0_col7, #T_339c8_row0_col8, #T_339c8_row0_col9, #T_339c8_row1_col0, #T_339c8_row1_col1, #T_339c8_row1_col2, #T_339c8_row1_col3, #T_339c8_row1_col4, #T_339c8_row1_col5, #T_339c8_row1_col6, #T_339c8_row1_col7, #T_339c8_row1_col8, #T_339c8_row1_col9, #T_339c8_row2_col0, #T_339c8_row2_col1, #T_339c8_row2_col2, #T_339c8_row2_col3, #T_339c8_row2_col4, #T_339c8_row2_col5, #T_339c8_row2_col6, #T_339c8_row2_col7, #T_339c8_row2_col8, #T_339c8_row2_col9, #T_339c8_row3_col0, #T_339c8_row3_col1, #T_339c8_row3_col2, #T_339c8_row3_col3, #T_339c8_row3_col4, #T_339c8_row3_col5, #T_339c8_row3_col6, #T_339c8_row3_col7, #T_339c8_row3_col8, #T_339c8_row3_col9, #T_339c8_row4_col0, #T_339c8_row4_col1, #T_339c8_row4_col2, #T_339c8_row4_col3, #T_339c8_row4_col4, #T_339c8_row4_col5, #T_339c8_row4_col6, #T_339c8_row4_col7, #T_339c8_row4_col8, #T_339c8_row4_col9, #T_339c8_row5_col0, #T_339c8_row5_col1, #T_339c8_row5_col2, #T_339c8_row5_col3, #T_339c8_row5_col4, #T_339c8_row5_col5, #T_339c8_row5_col6, #T_339c8_row5_col7, #T_339c8_row5_col8, #T_339c8_row5_col9, #T_339c8_row7_col0, #T_339c8_row7_col1, #T_339c8_row7_col2, #T_339c8_row7_col3, #T_339c8_row7_col4, #T_339c8_row7_col5, #T_339c8_row7_col6, #T_339c8_row7_col7, #T_339c8_row7_col8, #T_339c8_row7_col9, #T_339c8_row8_col0, #T_339c8_row8_col1, #T_339c8_row8_col2, #T_339c8_row8_col3, #T_339c8_row8_col4, #T_339c8_row8_col5, #T_339c8_row8_col6, #T_339c8_row8_col7, #T_339c8_row8_col8, #T_339c8_row8_col9, #T_339c8_row9_col0, #T_339c8_row9_col1, #T_339c8_row9_col2, #T_339c8_row9_col3, #T_339c8_row9_col4, #T_339c8_row9_col5, #T_339c8_row9_col6, #T_339c8_row9_col7, #T_339c8_row9_col8, #T_339c8_row9_col9 {\n",
       "  background-color: lightblue;\n",
       "  color: black;\n",
       "  border-color: white;\n",
       "  border-style: solid;\n",
       "  border-width: 1px;\n",
       "  background-color: lightgreen;\n",
       "}\n",
       "#T_339c8_row6_col0, #T_339c8_row6_col1, #T_339c8_row6_col2, #T_339c8_row6_col3, #T_339c8_row6_col4, #T_339c8_row6_col5, #T_339c8_row6_col6, #T_339c8_row6_col7, #T_339c8_row6_col8, #T_339c8_row6_col9 {\n",
       "  background-color: lightblue;\n",
       "  color: black;\n",
       "  border-color: white;\n",
       "  border-style: solid;\n",
       "  border-width: 1px;\n",
       "  background-color: lightcoral;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_339c8\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_339c8_level0_col0\" class=\"col_heading level0 col0\" >Query</th>\n",
       "      <th id=\"T_339c8_level0_col1\" class=\"col_heading level0 col1\" >Correctness (Qwen)</th>\n",
       "      <th id=\"T_339c8_level0_col2\" class=\"col_heading level0 col2\" >Readability (Qwen)</th>\n",
       "      <th id=\"T_339c8_level0_col3\" class=\"col_heading level0 col3\" >Best Practices (Qwen)</th>\n",
       "      <th id=\"T_339c8_level0_col4\" class=\"col_heading level0 col4\" >Correctness (Claude)</th>\n",
       "      <th id=\"T_339c8_level0_col5\" class=\"col_heading level0 col5\" >Readability (Claude)</th>\n",
       "      <th id=\"T_339c8_level0_col6\" class=\"col_heading level0 col6\" >Best Practices (Claude)</th>\n",
       "      <th id=\"T_339c8_level0_col7\" class=\"col_heading level0 col7\" >Overall Score (Qwen)</th>\n",
       "      <th id=\"T_339c8_level0_col8\" class=\"col_heading level0 col8\" >Overall Score (Claude)</th>\n",
       "      <th id=\"T_339c8_level0_col9\" class=\"col_heading level0 col9\" >Winner</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_339c8_row0_col0\" class=\"data row0 col0\" >Build an MCP server that fetches the latest tweets from a user and sends them as a daily digest email via Gmail.</td>\n",
       "      <td id=\"T_339c8_row0_col1\" class=\"data row0 col1\" >0.914</td>\n",
       "      <td id=\"T_339c8_row0_col2\" class=\"data row0 col2\" >0.903</td>\n",
       "      <td id=\"T_339c8_row0_col3\" class=\"data row0 col3\" >0.710</td>\n",
       "      <td id=\"T_339c8_row0_col4\" class=\"data row0 col4\" >0.738</td>\n",
       "      <td id=\"T_339c8_row0_col5\" class=\"data row0 col5\" >0.900</td>\n",
       "      <td id=\"T_339c8_row0_col6\" class=\"data row0 col6\" >0.758</td>\n",
       "      <td id=\"T_339c8_row0_col7\" class=\"data row0 col7\" >0.842</td>\n",
       "      <td id=\"T_339c8_row0_col8\" class=\"data row0 col8\" >0.799</td>\n",
       "      <td id=\"T_339c8_row0_col9\" class=\"data row0 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_339c8_row1_col0\" class=\"data row1 col0\" >Build an MCP server that creates a new Notion page when someone drops a file into a specific Google Drive folder.</td>\n",
       "      <td id=\"T_339c8_row1_col1\" class=\"data row1 col1\" >0.935</td>\n",
       "      <td id=\"T_339c8_row1_col2\" class=\"data row1 col2\" >0.910</td>\n",
       "      <td id=\"T_339c8_row1_col3\" class=\"data row1 col3\" >0.779</td>\n",
       "      <td id=\"T_339c8_row1_col4\" class=\"data row1 col4\" >0.890</td>\n",
       "      <td id=\"T_339c8_row1_col5\" class=\"data row1 col5\" >0.899</td>\n",
       "      <td id=\"T_339c8_row1_col6\" class=\"data row1 col6\" >0.734</td>\n",
       "      <td id=\"T_339c8_row1_col7\" class=\"data row1 col7\" >0.874</td>\n",
       "      <td id=\"T_339c8_row1_col8\" class=\"data row1 col8\" >0.841</td>\n",
       "      <td id=\"T_339c8_row1_col9\" class=\"data row1 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_339c8_row2_col0\" class=\"data row2 col0\" >Create a small MCP server that listens for incoming questions from Discord and uses OpenAI to generate answers.</td>\n",
       "      <td id=\"T_339c8_row2_col1\" class=\"data row2 col1\" >0.920</td>\n",
       "      <td id=\"T_339c8_row2_col2\" class=\"data row2 col2\" >0.904</td>\n",
       "      <td id=\"T_339c8_row2_col3\" class=\"data row2 col3\" >0.755</td>\n",
       "      <td id=\"T_339c8_row2_col4\" class=\"data row2 col4\" >0.894</td>\n",
       "      <td id=\"T_339c8_row2_col5\" class=\"data row2 col5\" >0.905</td>\n",
       "      <td id=\"T_339c8_row2_col6\" class=\"data row2 col6\" >0.773</td>\n",
       "      <td id=\"T_339c8_row2_col7\" class=\"data row2 col7\" >0.860</td>\n",
       "      <td id=\"T_339c8_row2_col8\" class=\"data row2 col8\" >0.857</td>\n",
       "      <td id=\"T_339c8_row2_col9\" class=\"data row2 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_339c8_row3_col0\" class=\"data row3 col0\" >Build an MCP server that watches a GitHub repo for new issues and posts them into a Telegram group.</td>\n",
       "      <td id=\"T_339c8_row3_col1\" class=\"data row3 col1\" >0.917</td>\n",
       "      <td id=\"T_339c8_row3_col2\" class=\"data row3 col2\" >0.913</td>\n",
       "      <td id=\"T_339c8_row3_col3\" class=\"data row3 col3\" >0.813</td>\n",
       "      <td id=\"T_339c8_row3_col4\" class=\"data row3 col4\" >0.741</td>\n",
       "      <td id=\"T_339c8_row3_col5\" class=\"data row3 col5\" >0.905</td>\n",
       "      <td id=\"T_339c8_row3_col6\" class=\"data row3 col6\" >0.796</td>\n",
       "      <td id=\"T_339c8_row3_col7\" class=\"data row3 col7\" >0.881</td>\n",
       "      <td id=\"T_339c8_row3_col8\" class=\"data row3 col8\" >0.814</td>\n",
       "      <td id=\"T_339c8_row3_col9\" class=\"data row3 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_339c8_row4_col0\" class=\"data row4 col0\" >Write an MCP agent that reads a URL, extracts metadata with BeautifulSoup, and logs it to a CSV file.</td>\n",
       "      <td id=\"T_339c8_row4_col1\" class=\"data row4 col1\" >0.901</td>\n",
       "      <td id=\"T_339c8_row4_col2\" class=\"data row4 col2\" >0.906</td>\n",
       "      <td id=\"T_339c8_row4_col3\" class=\"data row4 col3\" >0.770</td>\n",
       "      <td id=\"T_339c8_row4_col4\" class=\"data row4 col4\" >0.762</td>\n",
       "      <td id=\"T_339c8_row4_col5\" class=\"data row4 col5\" >0.904</td>\n",
       "      <td id=\"T_339c8_row4_col6\" class=\"data row4 col6\" >0.754</td>\n",
       "      <td id=\"T_339c8_row4_col7\" class=\"data row4 col7\" >0.859</td>\n",
       "      <td id=\"T_339c8_row4_col8\" class=\"data row4 col8\" >0.807</td>\n",
       "      <td id=\"T_339c8_row4_col9\" class=\"data row4 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "      <td id=\"T_339c8_row5_col0\" class=\"data row5 col0\" >Create an MCP endpoint that receives a YouTube link, downloads the audio, and sends it via email.</td>\n",
       "      <td id=\"T_339c8_row5_col1\" class=\"data row5 col1\" >0.932</td>\n",
       "      <td id=\"T_339c8_row5_col2\" class=\"data row5 col2\" >0.908</td>\n",
       "      <td id=\"T_339c8_row5_col3\" class=\"data row5 col3\" >0.731</td>\n",
       "      <td id=\"T_339c8_row5_col4\" class=\"data row5 col4\" >0.900</td>\n",
       "      <td id=\"T_339c8_row5_col5\" class=\"data row5 col5\" >0.910</td>\n",
       "      <td id=\"T_339c8_row5_col6\" class=\"data row5 col6\" >0.713</td>\n",
       "      <td id=\"T_339c8_row5_col7\" class=\"data row5 col7\" >0.857</td>\n",
       "      <td id=\"T_339c8_row5_col8\" class=\"data row5 col8\" >0.841</td>\n",
       "      <td id=\"T_339c8_row5_col9\" class=\"data row5 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "      <td id=\"T_339c8_row6_col0\" class=\"data row6 col0\" >Create an MCP server that takes a PDF uploaded via a web form, summarizes it using an LLM, and returns the summary.</td>\n",
       "      <td id=\"T_339c8_row6_col1\" class=\"data row6 col1\" >0.920</td>\n",
       "      <td id=\"T_339c8_row6_col2\" class=\"data row6 col2\" >0.905</td>\n",
       "      <td id=\"T_339c8_row6_col3\" class=\"data row6 col3\" >0.703</td>\n",
       "      <td id=\"T_339c8_row6_col4\" class=\"data row6 col4\" >0.845</td>\n",
       "      <td id=\"T_339c8_row6_col5\" class=\"data row6 col5\" >0.908</td>\n",
       "      <td id=\"T_339c8_row6_col6\" class=\"data row6 col6\" >0.802</td>\n",
       "      <td id=\"T_339c8_row6_col7\" class=\"data row6 col7\" >0.843</td>\n",
       "      <td id=\"T_339c8_row6_col8\" class=\"data row6 col8\" >0.851</td>\n",
       "      <td id=\"T_339c8_row6_col9\" class=\"data row6 col9\" >Claude</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "      <td id=\"T_339c8_row7_col0\" class=\"data row7 col0\" >Write an MCP-compatible server that pulls job listings from LinkedIn and posts them to a Slack channel every hour.</td>\n",
       "      <td id=\"T_339c8_row7_col1\" class=\"data row7 col1\" >0.920</td>\n",
       "      <td id=\"T_339c8_row7_col2\" class=\"data row7 col2\" >0.780</td>\n",
       "      <td id=\"T_339c8_row7_col3\" class=\"data row7 col3\" >0.793</td>\n",
       "      <td id=\"T_339c8_row7_col4\" class=\"data row7 col4\" >0.694</td>\n",
       "      <td id=\"T_339c8_row7_col5\" class=\"data row7 col5\" >0.898</td>\n",
       "      <td id=\"T_339c8_row7_col6\" class=\"data row7 col6\" >0.708</td>\n",
       "      <td id=\"T_339c8_row7_col7\" class=\"data row7 col7\" >0.831</td>\n",
       "      <td id=\"T_339c8_row7_col8\" class=\"data row7 col8\" >0.767</td>\n",
       "      <td id=\"T_339c8_row7_col9\" class=\"data row7 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "      <td id=\"T_339c8_row8_col0\" class=\"data row8 col0\" >Build a server that monitors a subreddit for new posts, filters them by keyword, and logs them in a Notion database.</td>\n",
       "      <td id=\"T_339c8_row8_col1\" class=\"data row8 col1\" >0.919</td>\n",
       "      <td id=\"T_339c8_row8_col2\" class=\"data row8 col2\" >0.905</td>\n",
       "      <td id=\"T_339c8_row8_col3\" class=\"data row8 col3\" >0.718</td>\n",
       "      <td id=\"T_339c8_row8_col4\" class=\"data row8 col4\" >0.632</td>\n",
       "      <td id=\"T_339c8_row8_col5\" class=\"data row8 col5\" >0.906</td>\n",
       "      <td id=\"T_339c8_row8_col6\" class=\"data row8 col6\" >0.791</td>\n",
       "      <td id=\"T_339c8_row8_col7\" class=\"data row8 col7\" >0.847</td>\n",
       "      <td id=\"T_339c8_row8_col8\" class=\"data row8 col8\" >0.776</td>\n",
       "      <td id=\"T_339c8_row8_col9\" class=\"data row8 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_339c8_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "      <td id=\"T_339c8_row9_col0\" class=\"data row9 col0\" >Create a minimal MCP server that extracts tabular data from a URL and sends it to an Airtable base.</td>\n",
       "      <td id=\"T_339c8_row9_col1\" class=\"data row9 col1\" >0.920</td>\n",
       "      <td id=\"T_339c8_row9_col2\" class=\"data row9 col2\" >0.890</td>\n",
       "      <td id=\"T_339c8_row9_col3\" class=\"data row9 col3\" >0.860</td>\n",
       "      <td id=\"T_339c8_row9_col4\" class=\"data row9 col4\" >0.897</td>\n",
       "      <td id=\"T_339c8_row9_col5\" class=\"data row9 col5\" >0.905</td>\n",
       "      <td id=\"T_339c8_row9_col6\" class=\"data row9 col6\" >0.827</td>\n",
       "      <td id=\"T_339c8_row9_col7\" class=\"data row9 col7\" >0.890</td>\n",
       "      <td id=\"T_339c8_row9_col8\" class=\"data row9 col8\" >0.876</td>\n",
       "      <td id=\"T_339c8_row9_col9\" class=\"data row9 col9\" >Qwen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1380724e0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from typing import List, Dict, Any\n",
    "\n",
    "def create_results_table(results: List[Dict[str, Any]]) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Create a formatted table from evaluation results\n",
    "    \"\"\"\n",
    "    # Create DataFrame from results\n",
    "    df = pd.DataFrame(results)\n",
    "    \n",
    "    # Calculate overall scores (average of the three metrics)\n",
    "    df['overall_score_qwen'] = (df['Correctness_Qwen'] + df['readability_Qwen'] + df['bestpractices_Qwen']) / 3\n",
    "    df['overall_score_claude'] = (df['Correctness_Sonnet'] + df['readability_Sonnet'] + df['bestpractices_Sonnet']) / 3\n",
    "    \n",
    "    # Determine winner\n",
    "    df['winner'] = df.apply(lambda row: 'Qwen' if row['overall_score_qwen'] > row['overall_score_claude'] else 'Claude', axis=1)\n",
    "    \n",
    "    # Round all numeric columns to 3 decimal places\n",
    "    numeric_columns = [\n",
    "        'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n",
    "        'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n",
    "        'overall_score_qwen', 'overall_score_claude'\n",
    "    ]\n",
    "    \n",
    "    for col in numeric_columns:\n",
    "        if col in df.columns:\n",
    "            df[col] = df[col].round(3)\n",
    "    \n",
    "    # Select and rename columns for the final table\n",
    "    final_df = df[[\n",
    "        'query',\n",
    "        'Correctness_Qwen', 'readability_Qwen', 'bestpractices_Qwen',\n",
    "        'Correctness_Sonnet', 'readability_Sonnet', 'bestpractices_Sonnet',\n",
    "        'overall_score_qwen', 'overall_score_claude', 'winner'\n",
    "    ]].copy()\n",
    "    \n",
    "    # Rename columns for better readability\n",
    "    final_df.columns = [\n",
    "        'Query',\n",
    "        'Correctness (Qwen)', 'Readability (Qwen)', 'Best Practices (Qwen)',\n",
    "        'Correctness (Claude)', 'Readability (Claude)', 'Best Practices (Claude)',\n",
    "        'Overall Score (Qwen)', 'Overall Score (Claude)', 'Winner'\n",
    "    ]\n",
    "    \n",
    "    return final_df\n",
    "\n",
    "def display_results_table(results: List[Dict[str, Any]]):\n",
    "    \"\"\"\n",
    "    Display the results table with formatting\n",
    "    \"\"\"\n",
    "    df = create_results_table(results)\n",
    "    \n",
    "    # Display the table\n",
    "    print(\"=== EVALUATION RESULTS TABLE ===\")\n",
    "    print(df.to_string(index=False))\n",
    "    \n",
    "    # Print summary statistics\n",
    "    print(\"\\n=== SUMMARY STATISTICS ===\")\n",
    "    print(f\"Total queries: {len(df)}\")\n",
    "    print(f\"Qwen wins: {len(df[df['Winner'] == 'Qwen'])}\")\n",
    "    print(f\"Claude wins: {len(df[df['Winner'] == 'Claude'])}\")\n",
    "    \n",
    "    # Average scores\n",
    "    print(f\"\\nAverage Overall Scores:\")\n",
    "    print(f\"Qwen: {df['Overall Score (Qwen)'].mean():.3f}\")\n",
    "    print(f\"Claude: {df['Overall Score (Claude)'].mean():.3f}\")\n",
    "    \n",
    "    return df\n",
    "\n",
    "# If you want to display in Jupyter with better formatting:\n",
    "def display_formatted_table(results: List[Dict[str, Any]]):\n",
    "    \"\"\"\n",
    "    Display formatted table with styling (for Jupyter notebooks)\n",
    "    \"\"\"\n",
    "    df = create_results_table(results)\n",
    "    \n",
    "    # Apply styling\n",
    "    styled_df = df.style.set_properties(**{\n",
    "        'background-color': 'lightblue',\n",
    "        'color': 'black',\n",
    "        'border-color': 'white',\n",
    "        'border-style': 'solid',\n",
    "        'border-width': '1px'\n",
    "    }).format({\n",
    "        'Correctness (Qwen)': '{:.3f}',\n",
    "        'Readability (Qwen)': '{:.3f}',\n",
    "        'Best Practices (Qwen)': '{:.3f}',\n",
    "        'Correctness (Claude)': '{:.3f}',\n",
    "        'Readability (Claude)': '{:.3f}',\n",
    "        'Best Practices (Claude)': '{:.3f}',\n",
    "        'Overall Score (Qwen)': '{:.3f}',\n",
    "        'Overall Score (Claude)': '{:.3f}'\n",
    "    }).apply(lambda x: ['background-color: lightgreen' if x['Winner'] == 'Qwen' else 'background-color: lightcoral' for i in range(len(x))], axis=1)\n",
    "    \n",
    "    return styled_df\n",
    "\n",
    "styled_table = display_formatted_table(results)\n",
    "styled_table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
